package fetcher

import (
	"bufio"
	"crypto/tls"
	"io/ioutil"
	"log"
	"net/http"
	"time"

	"github.com/pkg/errors"
	"golang.org/x/net/html/charset"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/unicode"
	"golang.org/x/text/transform"
)

//限速
var rateLimiter = time.Tick(10 * time.Millisecond)

var urls = make(map[string]bool)

func Fetch(url string) ([]byte, error) {
	<-rateLimiter

	if repeatRequest(url) == true {
		return nil, errors.New("repeat request")
	}

	req, err := getRequest(url)

	if err != nil {
		return nil, err
	}

	urls[url] = true

	resp, err := getClient().Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		log.Printf("fetch url: %s, status code: %d", url, resp.StatusCode)
	}

	bodyReader := bufio.NewReader(resp.Body)
	e := determineEncoding(bodyReader)
	utf8Reader := transform.NewReader(bodyReader, e.NewDecoder())

	return ioutil.ReadAll(utf8Reader)
}

//判断URL是否已经请求过了, 避免重复请求
func repeatRequest(url string) bool {
	if urls[url] == true {
		return true
	}

	return false
}

func getRequest(url string) (*http.Request, error) {
	request, err := http.NewRequest(http.MethodGet, url, nil)

	if err != nil {
		return nil, err
	}

	request.Header.Add("User-Agent",
		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36")
	request.Header.Add("Cookie",
		"JSESSIONID=ABAAABAAADEAAFI33156134DA74080EC1207A59639934BA; _ga=GA1.2.2122530788.1524621866; _gid=GA1.2.787780627.1524621866; index_location_city=%E5%85%A8%E5%9B%BD; user_trace_token=20180425100401-5825ea7b-a0fe-423e-a499-fd22fa137c8d; LGUID=20180425101647-b4dc1466-482e-11e8-a317-525400f775ce; Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1524622643; LGSID=20180425120924-708f0085-483e-11e8-bad8-5254005c3644; PRE_UTM=; PRE_HOST=; PRE_SITE=; PRE_LAND=https%3A%2F%2Fwww.lagou.com%2Fzhaopin%2Fshujuyunying%2F; X_HTTP_TOKEN=767cc0f2c6f275b6bf4e9633c938725d; ab_test_random_num=0; TG-TRACK-CODE=index_navigation; SEARCH_ID=97268ea04ff8426bab3fa6fb61d8d649; LGRID=20180425123027-60fe2756-4841-11e8-a330-525400f775ce; Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1524630663")
	return request, nil
}

func getClient() *http.Client {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	client := &http.Client{Transport: tr}
	return client
}

func determineEncoding(r *bufio.Reader) encoding.Encoding {
	bytes, err := bufio.NewReader(r).Peek(1024)

	if err != nil {
		log.Printf("fetcher error %v", err)
		return unicode.UTF8
	}
	e, _, _ := charset.DetermineEncoding(bytes, "")
	return e
}
